#!/usr/bin/env python2
#coding:utf8

import commands
import os
import sys
import socket
import time
import errno
import json
import getopt
import random
import re
import copy
import string
import shutil
import threading
import fcntl
import traceback
import pprint

from buddha.snapshot import _snapshot_manage
from buddha.fileutil import LichFile

from config import Config
from env import Env
from instence import Instence
from storage import Storage
from disk_manage import DiskManage
from utils import Exp, _dmsg, _dwarn, _derror, _lock_file, _lock_file1, _unlock_file1,\
    _syserror, _syswarn, _sysinfo,  _get_allip, _remove_ip, _exchange_mask, \
    etchosts_update, cpu_usage, etchosts_delete, _install_init,\
    _exec_shell, _exec_pipe, mutil_exec, _exec_pipe1, _exec_system,\
    _human_readable, _human_unreadable, _str2dict, timeit, _get_value, _set_value, _check_config,\
    _exec_shell1
from instence_scan import InstenceScan
from metabalance import balance_all
from instence_drop import InstenceDrop
from node_drop import NodeDrop
from ucarp import Ucarp

from metanode_balance_with_diskmap import balance
from cache_manage import CommonCache
from intelcas_config import read_cache_section, read_core_section
from recovery import NodeStat
from etcd_manage import Etcd_manage
#import task

from tlog import tlog

CACHE_VERSION_STANDARD = 0
CACHE_VERSION_FINGERPRINT = 1

class Node(object):
    def __init__(self):
        self.config = Config()
        self.disk_manage = DiskManage(self)
        self.env = Env(self.config, self.disk_manage)
        self.instences = []
        self.ttyonly = False
        #self.cmd = "sudo -u %s -g %s " % (self.config.user, self.config.group) + self.config.lich + "/sbin/lichd"
        #self.cmd = "sudo -u %s " % (self.config.user) + self.config.lich + "/sbin/lichd"
        self.cmd =self.config.lich + "/sbin/lichd"

        self.instences.append(Instence(self.config))
        self.storage = Storage(self.config)
        self.threads = []

        self.lich_admin = os.path.join(self.config.lich, "libexec/lich.admin")
        self.commoncache = CommonCache(self)

        self.__startstop_lock = "/var/run/fusionstack_nodestartstop.lock"

    def init(self):
        self.env.set_all()
        os.system("rm -rf %s" % self.config.shm)

        def instance_init_warp(instance, idx):
            try:
                instance.init(self.config.clustername, idx)
            except Exp, e:
                if (e.errno == errno.EEXIST):
                    _dwarn(e.err)
                else:
                    raise e

        args = []
        idx = 0
        for i in self.instences:
            args.append([i, idx])
            idx = idx + 1
        mutil_exec(instance_init_warp, args)
        self.ln()

        if not self.config.testing:
            _install_init(self.config.home)

        ucarp = Ucarp(self)
        ucarp.init()

    def stat(self):
        stat = {}
        for i in self.instences:
            stat[i.name] = i.running()
        return stat

    def stat_json(self):
        lst = []
        for i in self.instences:
            lst = lst + [i.show()]
        return lst

    def run_python_file(self, cmd):
        _cmd = 'python %s/%s' % (self.config.home, cmd)
        os.system(_cmd)

    def __stop_nolock(self, service = None, op = None):
        if service is not None:
            i = Instence(self.config, service)
            i.stop(self.ttyonly)
            return 0

        ucarp = Ucarp(self)
        ucarp.stop()

        #cmd = self.config.home + '/lich/admin/fusionstack_shell.py --stop'
        #os.system(cmd)
        self.run_python_file('lich/admin/lich_dump.py --stop')
        self.run_python_file('lich/admin/lich_cron.py --stop')

        ts = []
        def instance_stop_warp(i):
            i.stop(self.ttyonly)

        args = [[x] for x in self.instences]
        mutil_exec(instance_stop_warp, args)
        self.env.unset_crontab()

        #bug 10507 暂时把移动数据放到开机和关机
        #self.__mv_stuff_nolock(op)
        #bug 10507 暂时把移动数据放到开机和关机

        #os.system("systemctl stop etcd")
        self.run_python_file('lich/admin/lich_arp.py --stop')

        os.system("for i in `ps -ef | grep 'check_client_ip' | grep -v grep | awk '{print $2}'`; do kill -9 $i; done")

    def stop(self, service = None, op = None):
        self.__stop_nolock(service, op)

    ##
    # 判断文件内容是否是dirty，如果是dirty，抛异常
    # 如果文件不存在，也抛异常
    # @Param path
    #
    # @Returns
    def __check_mv_clean(self, path = None):
        s = ''
        with open(path, 'r') as f:
            s = f.read().strip()
        if s != "clean":
            raise Exp(errno.EPERM, "%s %s not clean" % (path, s))

    def __set_mv_dirty(self, path = None):
        with open(path, 'w') as f:
            f.write("dirty")
            os.fsync(f.fileno())

    def __set_mv_clean(self, path = None):
        with open(path, 'w') as f:
            f.write("clean")
            os.fsync(f.fileno())

    def __mv_clock_nolock(self, op = None):
        path = "%s/data/clock_status" % (self.config.home)

        start1 = "mkdir -p %s; rm -rf %s/clock; rm -rf %s/clock.tmp; mv %s/data/clock %s/clock.tmp; sync;" % \
                (self.config.shm, self.config.shm, self.config.shm, self.config.home, self.config.shm)
        start2 = "mv %s/clock.tmp %s/clock && sync && rm -rf %s/data/clock" % \
                (self.config.shm, self.config.shm, self.config.home)

        stop1 = "rm -rf %s/data/clock && rm -rf %s/data/clock.tmp && mv %s/clock %s/data/clock.tmp && sync " % \
                (self.config.home, self.config.home, self.config.shm, self.config.home)
        stop2 = "mv %s/data/clock.tmp %s/data/clock && sync && rm -rf %s/clock && rm -rf %s/clock.tmp" % \
                (self.config.home, self.config.home, self.config.shm, self.config.shm)

        cleandisk = "rm -rf %s/data/clock" % (self.config.home)

        if op == '--start':
            if not os.path.exists("%s/data/clock" % self.config.home):
                return
            #_dmsg("%s; %s" % (start1, start2))
            try:
                self.__check_mv_clean(path)
            except Exception, e:
                os.system(cleandisk)
                raise Exp(errno.EPERM, "clock dirty %s" % (e))

            self.__set_mv_dirty(path)
            os.system(start1)
            os.system(start2)
            self.__set_mv_clean(path)
        elif op == '--stop':
            if not os.path.exists("%s/clock" % self.config.shm):
                return
            #_dmsg(" %s; %s" % (stop1, stop2))
            self.__set_mv_dirty(path)
            os.system(stop1)
            os.system(stop2)
            self.__set_mv_clean(path)
        else:
            return

    def __mv_hsm_nolock(self, op = None):
        path = "%s/data/shm_status" % (self.config.home)

        start1 = "mkdir -p %s; rm -rf %s/hsm; rm -rf %s/hsm.tmp; mv %s/data/hsm %s/hsm.tmp; sync;" % \
                (self.config.shm, self.config.shm, self.config.shm, self.config.home, self.config.shm)
        start2 = "mv %s/hsm.tmp %s/hsm && sync" % (self.config.shm, self.config.shm)

        stop1 = "rm -rf %s/data/hsm && rm -rf %s/data/hsm.tmp && mv %s/hsm %s/data/hsm.tmp && sync " % \
                (self.config.home, self.config.home, self.config.shm, self.config.home)
        stop2 = "mv %s/data/hsm.tmp %s/data/hsm && sync" % (self.config.home, self.config.home)

        cleandisk = "rm -rf %s/data/hsm" % (self.config.home)

        if op == '--start':
            if not os.path.exists("%s/data/hsm" % self.config.home):
                return

            #_dmsg("%s; %s" % (start1, start2))
            try:
                self.__check_mv_clean(path)
            except Exception, e:
                os.system(cleandisk)
                raise Exp(errno.EPERM, "shm dirty %s" % (e))

            self.__set_mv_dirty(path)
            os.system(start1)
            os.system(start2)
            self.__set_mv_clean(path)
        elif op == '--stop':
            if not os.path.exists("%s/hsm" % self.config.shm):
                return

            #_dmsg("%s; %s" % (stop1, stop2))
            self.__set_mv_dirty(path)
            os.system(stop1)
            os.system(stop2)
            self.__set_mv_clean(path)
        else:
            return

    def __mv_stuff_nolock(self, op = None):
        #bug 10507 暂时把移动数据放到开机和关机
        #先不移动，这个方案有问题
        #return None
        #bug 10507 暂时把移动数据放到开机和关机

        if self.config.home != '/opt/fusionstack' or self.config.shm != '/dev/shm/lich4':
            return

        self.__mv_clock_nolock(op)
        print 'mv clock success'
        self.__mv_hsm_nolock(op)
        print 'mv hsm success'

    def __lock_status(self):
        path = self.config.home + "/data/status/status"

        try:
            fd = open(path, 'r')
        except IOError as err:
            if err.errno == errno.ENOENT:
                fd = open(path, 'w')
            else:
                raise

        retry = 0
        while(1):
            try:
                fcntl.flock(fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
                break
            except IOError as err:
                if err.errno == errno.EAGAIN and retry < 3:
                    time.sleep(2)
                    retry += 1
                    continue
                else:
                    raise

        return fd

    def __unlock_status(self, fd):
        fcntl.flock(fd.fileno(), fcntl.LOCK_UN)
        fd.close()

    def disk2shm(self, service = None, op = None):
        tlog.write(ctx='disk2shm', msg='mv clock hsm from disk to shm')
        op = "--start"

        self.disk_manage.ensure_meta()

        lock_fd = self.__lock_status()
        try:
            self.__mv_stuff_nolock(op)
        except Exception, e:
            raise
        finally:
            self.__unlock_status(lock_fd)

    def shm2disk(self, service = None, op = None):
        print 'mv clock hsm from shm to disk'
        op = "--stop"

        lock_fd = self.__lock_status()
        try:
            self.__mv_stuff_nolock(op)
        except Exception, e:
            raise
        finally:
            self.__unlock_status(lock_fd)

    def check_cache_version(self):
        verpath = os.path.join(self.config.home, "data/status/cache_version")
        if os.path.exists(verpath):
            ver = int(_get_value(verpath))
        else:
            ver = CACHE_VERSION_STANDARD
            _set_value(verpath, str(ver))

        if ver == CACHE_VERSION_STANDARD:
            sqlite3 = os.path.join(self.config.home, "lich/admin/sqlite3.sh")
            dirname = os.path.join(self.config.home, "data/chunk/")
            if not os.path.exists(dirname):
                return

            for name in os.listdir(dirname):
                m = re.match('\d\.db$', name)
                if m is None:
                    continue

                # add colume fingerprint
                path = os.path.join(dirname, name)
                cmd = '''%s -d %s -t metadata -o listcol | grep fingerprint > /dev/null;
                        if [ $? != 0 ]; then
                            %s -d %s -t metadata -o addcol fingerprint integer 0;
                        fi
                ''' % (sqlite3, path, sqlite3, path)
                _exec_system(cmd, False, False)

                cmd = '''%s -d %s -t raw -o listcol | grep fingerprint > /dev/null;
                        if [ $? != 0 ]; then
                            %s -d %s -t raw -o addcol fingerprint integer 0;
                        fi
                ''' % (sqlite3, path, sqlite3, path)
                _exec_system(cmd, False, False)

                # add colume writeback_disk
                cmd = '''%s -d %s -t metadata -o listcol | grep wbdisk > /dev/null;
                        if [ $? != 0 ]; then
                            %s -d %s -t metadata -o addcol wbdisk integer -1;
                        fi
                ''' % (sqlite3, path, sqlite3, path)
                _exec_system(cmd, False, False)

                cmd = '''%s -d %s -t raw -o listcol | grep wbdisk > /dev/null;
                        if [ $? != 0 ]; then
                            %s -d %s -t raw -o addcol wbdisk integer -1;
                        fi
                ''' % (sqlite3, path, sqlite3, path)
                _exec_system(cmd, False, False)

                # add colume pool
                cmd = '''%s -d %s -t metadata -o listcol | grep pool > /dev/null;
                        if [ $? != 0 ]; then
                            %s -d %s -t metadata -o addcol pool text 'default';
                        fi
                ''' % (sqlite3, path, sqlite3, path)
                _exec_system(cmd, False, False)

                cmd = '''%s -d %s -t raw -o listcol | grep pool > /dev/null;
                        if [ $? != 0 ]; then
                            %s -d %s -t raw -o addcol pool text 'default';
                        fi
                ''' % (sqlite3, path, sqlite3, path)
                _exec_system(cmd, False, False)

            ver = CACHE_VERSION_FINGERPRINT
            _set_value(verpath, str(ver))

    def __setup_cache(self):
        read_cache_section(self.intelcas)
        read_core_section(self.intelcas)

    def ensure_mibs(self):
        mib_path = '/usr/share/mibs/ietf'
        src_path = '%s/etc/mibs/ietf' % self.config.home
        if not os.path.exists(mib_path):
            os.system('mkdir -p /usr/share/mibs')
            os.system('ln -s %s %s' % (src_path, mib_path))

        for mib in ['SNMPv2-MIB', 'HOST-RESOURCES-MIB', 'ENTITY-MIB']:
            if not os.path.isfile(os.path.join(mib_path, mib)):
                os.system('cp -f %s/%s %s' % (src_path, mib, mib_path))

        return True

        
    def __start_redis(self):
        for i in range(self.config.redis_group):
            home = os.path.abspath(self.config.home)
            redis = "%s/data/redis/%u" % (home, i) 
            cmd = "mkdir -p %s" % (redis)
            #print cmd
            os.system(cmd)
            cmd = "redis-server %s/etc/redis.conf  --dir %s --port %u --pidfile %s/redis.pid --unixsocket %s/redis.sock --logfile %s/redis_%u.log" % (home, redis, 6479 + i, redis, redis, home, i)
            #print cmd
            os.system(cmd)
        
        
    def __start_nolock(self, service = None, op = None):
        if (self.config.kv_engine == 'redis'):
            self.__start_redis()
        tlog.write(ctx='__start_nolock')
        # self.ensure_mibs()
        #self.__setup_cache()
        self.disk_manage.disk_load()
        self.disk_manage.disk_check('cacheset')

        self.check_cache_version()

        #bug 10507 暂时把移动数据放到开机和关机
        #self.__mv_stuff_nolock(op)
        #bug 10507 暂时把移动数据放到开机和关机

        if service is not None:
            i = Instence(self.config, service)
            i.start(self.ttyonly)
            return 0

        #os.system("systemctl start etcd")
        Etcd_manage.etcd_service_start_force()

        self.env.set_all()

        #cmd = self.config.home + '/lich/admin/fusionstack_shell.py --start'
        #os.system(cmd)

        self.run_python_file('lich/admin/lich_dump.py --start')
        self.run_python_file('lich/admin/lich_cron.py --start')

        cmd = "mkdir  %s 2>/dev/null" % self.config.shm
        os.system(cmd)

        #cmd = "cat /boot/config-`uname -r` | grep '^CONFIG_HZ=' | awk -F '=' '{print $2}' > %s/config_hz" % self.config.shm
        #os.system(cmd)

        ucarp = Ucarp(self)
        ucarp.start()

        ts = []
        def instance_start_warp(i):
            i.start(self.ttyonly)

        args = [[x] for x in self.instences]
        mutil_exec(instance_start_warp, args)

        self.run_python_file('lich/admin/lich_arp.py --start')

        self.commoncache.scan_cache_log()
        self.commoncache.reset_node_cache_policy()

    def start(self, service = None, op = None):
        self.__start_nolock(service, op)

    def dmsg(self, msg):
        sys.stdout.write(msg)
        sys.stdout.flush()

    def __get_lichdpid(self, lichd_no):
        pids = []
        pids.append(self.instences[lichd_no].ppid)
        pids.append(self.instences[lichd_no].pid)
        return pids

    def _print_stat(self, stat):
        if (stat['running'] or stat['starting']):
            for (k, v) in stat.items():
                _dwarn("%s:%s" % (k, str(v)))
        else:
            for (k, v) in stat.items():
                _derror("%s:%s" % (k, str(v)))

    def show(self, verbose):
        if (verbose):
            if (len(self.instences) == 1):
                stat = self.instences[0].show();
                if (self.instences[0].deleting and self.instences[0].running()):
                    stat['status'] = stat['status'] + ',deleting'
                elif (self.instences[0].starting()):
                    stat['status'] = stat['status'] + ',starting'
                self._print_stat(stat)
            else:
                for i in self.instences:
                    _dmsg("instences %s" % (i.home))
                    stat = i.show();
                    self._print_stat(stat)
        else:
            if (len(self.instences) == 1):
                if (self.instences[0].deleting and self.instences[0].running()):
                    print("deleting")
                elif (self.instences[0].deleted):
                    print("deleted")
                elif (self.instences[0].running()):
                    print("running")
                elif (self.instences[0].starting()):
                    print("starting")
                else:
                    print("stopped")
            else:
                for i in self.instences:
                    _dmsg("instences %s" % (i.home))
                    if (i.deleting and i.running()):
                        print("deleting")
                    elif (i.deleted):
                        print("deleted")
                    elif (i.running()):
                        print("running")
                    elif (i.starting()):
                        print("starting")
                    else:
                        print("stopped")

    def show_json(self):
        lst = []
        for i in self.instences:
            lst = lst + [i.show()]

        print (json.dumps(lst))

    def check_offline(self, path, chkid):
        sumline = 0
        offline = 0

        cmd = os.path.join(self.config.lich, "libexec/lich.inspect")
        try:
            (out_msg, err_msg) = _exec_pipe1([cmd, '--stat', chkid, '-v', '-p'], 1, False)
        except Exp, e:
            if e.errno == errno.ENOENT:
                path = os.path.join(path, chkid)
                if os.path.isdir(path):
                    os.rmdir(path)
                elif os.path.isfile(path):
                    os.remove(path)
                return True
            out_msg = e.out

        for line in out_msg.splitlines():
            m = re.search('\*.* : ', line)
            if m is not None:
                sumline += 1
            m = re.search('\*.* : offline', line)
            if m is not None:
                offline += 1

        if offline < sumline:
            path = os.path.join(path, chkid)
            if os.path.isdir(path):
                os.rmdir(path)
            elif os.path.isfile(path):
                os.remove(path)
            return True

        return False

    def show_offline(self, is_show):
        path = os.path.join(self.config.shm, 'offline')
        off_line = {}
        if os.path.isdir(path):
            for chkid in os.listdir(path):
                chk_path = os.path.join(path, chkid)
                for rawid in os.listdir(chk_path):
                    if not self.check_offline(chk_path, rawid):
                        if chkid not in off_line.keys():
                            off_line[chkid] = []
                        off_line[chkid].append(rawid)
                if chkid not in off_line.keys():
                    if not self.check_offline(path, chkid):
                        off_line[chkid] = []

        if is_show:
            for chkid in off_line:
                print chkid
                for rawid in off_line[chkid]:
                    print "  " + rawid

    def check(self, fix=True, op = None):
        if (op == None):
            return self.env.set_all()

        if (fix):
            cmd = "self.env.check_%s(True)" % (op)
        else:
            cmd = "self.env.check_%s(False)" % (op)
        exec(cmd)

    def update(self, dist, src, backup):
        backparent = os.path.split(backup)
        cmd = "mkdir -p %s" % (backparent[0])
        os.system(cmd)
        tmp = os.path.abspath(self.config.home + "/../tmp")
        cmd = "rm -rf %s > /dev/null 2>&1" % (tmp)
        cmd = "mkdir -p %s > /dev/null 2>&1" % (tmp)
        cmd = cmd + " && tar xf %s -C %s >> /dev/null" % (src, tmp)
        cmd = cmd + " && mkdir -p %s" % (dist)
        if ('etc' in dist):
            cmd = cmd + " && if [ -d %s ]; then cd %s &&  tar czf %s * ; fi" % (dist, dist, backup)
        elif ('qemu' in dist) and os.path.exists(dist):
            ls = os.listdir(dist)
            if ls:
                cmd = cmd + " && if [ -d %s ]; then cd %s &&  tar czf %s * ; fi" % (dist, dist, backup)
        elif ('lich' in dist):
            cmd = cmd + " && if [ -d %s ]; then cd %s &&  tar czf %s * && rm -rf *; fi" % (dist, dist, backup)
        cmd = cmd + " && cp -rf %s/* %s;rm -rf %s" % (tmp, dist, tmp)
        #print(cmd)
        os.system(cmd)

        if not self.config.testing:
            _install_init(self.config.home)

        #refresh for ucarp
        self.config.refresh()
        ucarp = Ucarp(self)
        ucarp.init()

    def ln(self, force=True):
        bin_path = os.path.join(self.config.lich, 'bin')
        execs = [os.path.join(bin_path, x) for x in os.listdir(bin_path)]

        for e in execs:
            e_basename = os.path.basename(e)
            cmd = 'ln -s %s /usr/bin/%s'%(e, e_basename)
            if force:
                cmd = 'ln -fs %s /usr/bin/%s'%(e, e_basename)
            os.system(cmd)

    def metabalance(self, op="all"):
        lfile = "/var/run/fusionstack_metabalance.lock"
        lock = _lock_file(lfile)

        s = self.config.sysstat()
        n = s["node"]
        if (n[0] == 1 and n[1] == 2):
            _derror("online (%u/%u), exit" % (n[0], n[1]))
            exit(errno.ENOSPC)

        instence_admin = None
        for i in self.instences:
            if (i.running() == False):
                continue

            if i.isadmin():
                instence_admin = i

        if instence_admin is None:
            raise Exp(1, "The operation needs to be performed in admin node")
        else:
            balance_all()

    def chunkbalance(self, op="all", rand=False):
        lfile = "/var/run/fusionstack_chunkbalance.lock"
        lock = _lock_file(lfile)

        s = self.config.sysstat()
        n = s["node"]
        if (n[0] == 1 and n[1] == 2):
            _derror("online (%u/%u), exit" % (n[0], n[1]))
            exit(errno.ENOSPC)

        #if (self.config.localize == 0):
        #    return
        if (rand):
            slp = random.randrange(0, 60 * 60 * 3)
            left = slp
            while (left > 0):
                left = left - 60
                _dmsg("sleep %u left %u" % (slp, left))
                time.sleep(60)

        if (op == "all"):
            for i in self.instences:
                if (i.running() == False):
                    continue

                i.chunkbalance('rack')

                #drop = InstenceDrop(i.config, i.home)
                #while (1):
                #    try:
                #        drop.decrease()
                #    except Exp, e:
                #        if (e.errno == errno.EAGAIN):
                #            continue
                #        else:
                #            raise
                #    break
        #os.system('rm %s' % (lfile))

    def __scan_prep(self):
        os.system("rm -rf %s/maping/network" % self.config.shm)

        s = self.config.sysstat()
        n = s["node"]
        if (n[0] == 1 and n[1] == 2):
            msg = ("online (%u/%u), exit" % (n[0], n[1]))
            _derror(msg)
            raise Exp(errno.ENOSPC, msg)

    def __parse_recovery_info(self):
        immediately = os.path.join(self.config.shm, "nodectl/recovery/immediately")
        recover_info = os.path.join(self.config.shm, "nodectl/recovery/info")

        LichFile.write_int(immediately, 1)

        time.sleep(1)

        fr = open(recover_info, 'r')
        for line in fr.readlines():
            if "recovery" in line:
                dirty_total = line.split(":")[1]
            elif "success" in line:
                success_count = line.split(":")[1]
            elif "fail" in line:
                fail_count = line.split(":")[1]
            elif "lost" in line:
                lost_count = line.split(":")[1]
            elif "offline" in line:
                offline_count = line.split(":")[1]

        dirty_left = int(dirty_total) - int(success_count)
        if (dirty_left > 0):
            progress = float(success_count) / float(dirty_total) * 100
            print "dirty total:%d, success:%d(%d%%), fail:%d, lost:%d, offline: %d" % (int(dirty_total), int(success_count), int(progress), int(fail_count), int(lost_count), int(offline_count))
        else:
            print "node clean"

        fr.close()

    def scan(self):
        #print("verbose:" + str(verbose))
        lfile = "/var/run/fusionstack_scan.lock"
        lock = _lock_file(lfile)

        self.__scan_prep()

        self.__parse_recovery_info()

    def recover(self, op="all", verbose = False):
        #print("verbose:" + str(verbose))

        lfile = "/var/run/fusionstack_recover.lock"
        lock = _lock_file(lfile)

        os.system("rm -rf %s/maping/network" % self.config.shm)

        s = self.config.sysstat()
        n = s["node"]
        if (n[0] == 1 and n[1] == 2):
            _derror("online (%u/%u), exit" % (n[0], n[1]))
            exit(errno.ENOSPC)

        def instance_recover_warp(i):
            _dmsg("recover %s" % (i.home))
            if (i.running() == False):
                return

            try:
                instence_scan = InstenceScan(i)
                instence_scan.recover(verbose)
            except Exp, e:
                _derror("%ss" % (e.err))
                exit(e.errno)

        args = [[x] for x in self.instences]
        mutil_exec(instance_recover_warp, args)

        """
        if (op == "all"):
            for i in self.instences:
                if (i.running() == False):
                    continue

                try:
                    instence_scan = InstenceScan(i)
                    instence_scan.recover()
                except Exp, e:
                    _derror("%ss" % (e.err))
                    exit(e.errno)
                    """

        #os.system('rm %s' % (lfile))

#     def recover(self, op="all"):
#         lfile = self.config.home + "/recover_lock"
#         lock = _lock_file(lfile)
#
#         task_queue = []
#         task.setup_db('sqlite://')
#         for i in self.instences:
#             _dmsg("scan %s" % (i.home))
#             if (i.running() == False):
#                 return
#
#             try:
#                 instence_scan = InstenceScan(i)
#                 task_queue.append(instence_scan.recover())
#             except Exp, e:
#                 _derror("%ss" % (e.err))
#                 exit(e.errno)
#
#         while not all(task.is_complete(task_id) for task_id in task_queue):
#             task_id = task.request()
#             if task_id:
#                 eventlet.spawn_n(task.run, task_id)
#             eventlet.sleep(0)

    def recover_replica(self, op="all"):
        lfile = "/var/run/fusionstack_recover_replica.lock"
        lock = _lock_file(lfile)

        s = self.config.sysstat()
        n = s["node"]
        if (n[0] == 1 and n[1] == 2):
            _derror("online (%u/%u), exit" % (n[0], n[1]))
            exit(errno.ENOSPC)

        def instance_recover_replica_warp(i):
            _dmsg("recover_replica %s" % (i.home))
            if (i.running() == False):
                return

            try:
                instence_scan = InstenceScan(i)
                instence_scan.recover_replica()
            except Exp, e:
                _derror("%ss" % (e.err))
                exit(e.errno)

        args = [[x] for x in self.instences]
        mutil_exec(instance_recover_replica_warp, args)

    def __drop_node(self):
        s = self.config.sysstat()
        n = s["node"]
        if (n[1] == 1 or n[1] == 2):
            _derror("online (%u/%u), exit" % (n[0], n[1]))
            exit(errno.ENOSPC)

        count = 0
        lst = []
        for i in self.instences:
            if (i.skiped):
                _dwarn("node skiped")
                continue

            lst.append(InstenceDrop(i.config, i.home))

        time.sleep(10)

        for drop in lst:
            drop.drop()
            count = count + 1

        if (count == 0):
            exit(errno.EPERM)

        os.system("rm -rf %s" % self.config.shm)

        self.config.hosts_load()
        self.config.dropnode([self.config.hostname])
        cluster = os.path.join(self.config.lich, "admin/cluster.py")
        _exec_shell("%s update etc"%(cluster))

    def __drop_ins_force(self, instence):
        cmd = "%s --dropnode %s"%(instence.lich_admin, instence.name)
        try:
            _exec_shell(cmd)
        except Exp, e:
            if e.errno == errno.ENOENT:
                pass
            else:
                raise

        while True:
            found = False
            res = _exec_pipe(["ps", "-ef"], 0, False)
            for line in res.splitlines():
                m = re.search("/sbin/lichd --home " + instence.home, line)
                if m is not None:
                    pid = line.split()[1]
                    _dmsg("stop %s %s" % (instence.home, pid))
                    os.system("kill -9 " + pid)
                    found = True
            if not found:
                return

    def node_drop(self, diskid=None):
        lfile = "/var/run/drop.lock"
        lock = _lock_file(lfile)

        node_drop = NodeDrop(self)
        node_drop.check(diskid)
        node_drop.start(diskid)

    def setcron(self, s):
        if (s == '0'):
            self.env.unset_crontab()
        elif (s == '1'):
            self.env.check_crontab()

    def skip(self, s):
        if (s != '0' and s != '1'):
                raise Exp(errno.EINVAL, 'value:%s not support' % s)

        for i in self.instences:
            i.skip(s)

    def iscsi_vip_check(self, delay = 0, p = False):
        admin = ""
        out = ""

        if not self.config.iscsi_vip:
            if self.config.is_vipconfd() and p:
                _dwarn("check your iscsi vip conf or not in networks")
            return

        if delay:
            time.sleep(delay)

        allip = _get_allip()

        try:
            out, err = _exec_pipe1([self.config.admin, "--listnode", "-v"], 0, False)
        except Exp, e:
            pass

        for line in out.splitlines():
            m = re.search("(\S+):admin", line)
            if m is not None:
                admin = m.group(1)

        if admin is not None:
            if "/" not in self.config.iscsi_vip:
                _dwarn("check your iscsi vip conf format: xxx.xxx.xxx.xxx/xx")
                return

            if not self.config.nohosts:
                ip = socket.gethostbyname(admin)
            else:
                ip = admin

            mask = _exchange_mask(self.config.network()[0][-1])
            addr =  "%s/%d"% (ip, mask)

            if addr in allip:
                if self.config.iscsi_vip in allip:
                    return

                _exec_pipe([self.config.admin, "--setvip"], 0, False)
                _dmsg("set vip %s" % self.config.iscsi_vip)
                return

        _remove_ip(self.config.iscsi_vip)

    def ucarp_vip_check(self):
        ucarp = Ucarp(self)
        ucarp.check()

    def vip_check(self, delay = 0, p = False):
        self.iscsi_vip_check(delay, p)
        self.ucarp_vip_check()

    def ucarp_show(self):
        ucarp = Ucarp(self)
        ucarp.show()

    def log_backup(self, stime):
        for i in self.instences:
            i.log_backup(stime)

    def log_clean(self):
        for i in self.instences:
            i.log_clean()

    def log_collect(self, begin, end):
        try:
            tmp = begin
            begin = time.mktime(time.strptime(begin, "%Y-%m-%d %H:%M:%S"))
            end = time.mktime(time.strptime(end, "%Y-%m-%d %H:%M:%S"))
            now = time.time()
            if (begin > now or end < begin):
                _derror("Invalid begin time:(%s)" % tmp)
                exit(errno.EINVAL)
        except Exception, e:
            _derror("time data does not match format '%Y-%m-%d %H:%M:%S'")
            print 'such as: '
            print '     lich.node --log collect "2016-06-07 11:17:56" "2016-06-07 11:17:59"'
            exit(errno.EPERM)

        if begin > end:
            tmp = begin
            begin = end
            end = tmp

        for i in self.instences:
            i.log_collect(begin, end)

    def log_tail(self):
        res = ""
        for i in self.instences:
            res = res + i.log_tail()
        return res

    def log_info(self):
        #typeEncode = sys.getfilesystemencoding()
        log_info = {}
        log_path = os.path.join(self.config.home, "log")
        st = os.statvfs(log_path)
        log_info['available'] = st.f_bavail * st.f_frsize
        log_info['capacity'] = st.f_blocks * st.f_frsize
        log_info['filesize'] = {}
        for logfile in os.listdir(log_path):
            filename = os.path.join(log_path, logfile)
            if not os.path.isfile(filename):
                continue
            '''
            data = chardet.detect(logfile)
            if data.get('encoding') is  None:
                pass
            elif data.get('encoding').lower() == 'utf-8':
                logfile = logfile.decode('utf-8','ignore').encode(typeEncode)
            elif data.get('encoding').lower() == 'gb2312':
                logfile = logfile.decode('gb2312','ignore').encode(typeEncode)
            elif data.get('encoding').lower() == 'gbk':
                logfile = logfile.decode('gbk','ignore').encode(typeEncode)
            elif data.get('encoding').lower() == 'gb18030':
                logfile = logfile.decode('gb18030','ignore').encode(typeEncode)
            '''

            log_info['filesize'][logfile] = os.path.getsize(filename) * 1024

        return log_info

    def disk_used_in_db(self, diskid):
        has = 0
        for disk in range(10):
            cmd = 'sqlite3 %s/data/chunk/%s.db "select * from metadata where disk=%s"' % (self.config.home, disk, diskid)
            status, output = commands.getstatusoutput(cmd)
            if len(output) != 0:
                has = 1
                break

            cmd = 'sqlite3 %s/data/chunk/%s.db "select * from raw where disk=%s"' % (self.config.home, disk, diskid)
            status, output = commands.getstatusoutput(cmd)
            if len(output) != 0:
                has = 1
                break

        return has

    def disk_clean(self, force=False, p=True):
        """ TODO disk_clean

        :param force:
        :return:
        """
        stat = {}
        stat['nodestat'] = self.instences[0].show()

        if not stat['nodestat']['running']:
            #_dwarn('node offline, can not clean any disk')
            return

        diskstat = os.path.join(self.config.shm, "nodectl/diskstat")
        info_path = os.path.join(self.config.home, 'data/disk/info')

        has_error = 0
        msg_error = ''
        if os.path.exists(diskstat):
            for pool in os.listdir(diskstat):
                stat_pool = os.path.join(diskstat, pool)
                for disk in os.listdir(stat_pool):
                    disk_num = disk.split('.')[0]
                    stat_file = os.path.join(stat_pool, disk)
                    if not os.path.exists(stat_file):
                        continue
                    d = LichFile.read_dict(stat_file)
                    if d['online'] == '1':
                        continue

                    # TODO
                    if d['used'] == '1':
                        _exec_system("rm -rf %s" % os.path.join(info_path, "%s.info" % disk_num), p)
                    else:
                        if force:
                            used_in_db = self.disk_used_in_db(disk_num)
                            if used_in_db:
                                has_error = 1
                                msg = "disk: %s was offline, but used in db\n" % (disk_num)
                                msg_error = msg_error + msg
                            else:
                                _exec_system("rm -rf %s" % os.path.join(info_path, "%s.info" % disk_num), p)
                        else:
                            has_error = 1
                            msg = "disk: %s was offline, but used %s. need --force to clean it\n" % (disk_num, d['used'])
                            msg_error = msg_error + msg

        if has_error:
            raise Exp(1, msg_error)

    def _merge_recovery_info(self, pool_home):
        """ {
        'status': 'waiting',
        'lastscan': '1519097198',
        'recovery': '0',
        'offline': '0',
        'lost': '0',
        'check': '0',
        'success': '0',
        'fail': '0',
        'speed': '0'
        }

        :param pool_home:
        :return:
        """
        node_info_file = os.path.join(pool_home, "info")
        disk_info_file = os.path.join(pool_home, "disk_info")

        if os.path.exists(node_info_file):
            node_info = LichFile.read_dict(node_info_file)
        else:
            node_info = {}

        if os.path.exists(disk_info_file):
            disk_info = LichFile.read_dict(disk_info_file)
        else:
            disk_info = {}

        if len(node_info) == 0:
            return disk_info
        elif len(disk_info) == 0:
            return node_info

        d = {}
        for k, v in node_info.iteritems():
            if k == 'lastscan':
                # TODO
                # d[k] = min(int(v), int(disk_info[k]))
                pass
            elif k == 'status':
                if v == 'scanning' or disk_info[k] == 'scanning':
                    d[k] = 'scanning'
                elif v == 'running' or disk_info[k] == 'running':
                    d[k] = 'running'
                else:
                    d[k] = 'waiting'
            elif k in disk_info:
                d[k] = int(v) + int(disk_info[k])

        node_lastscan = int(node_info['lastscan'])
        disk_lastscan = int(disk_info['lastscan'])

        if node_info['status'] == 'waiting' and disk_info['status'] == 'waiting':
            d['lastscan'] = max(node_lastscan, disk_lastscan)
        elif node_info['status'] == 'waiting':
            d['lastscan'] = disk_lastscan
        elif disk_info['status'] == 'waiting':
            d['lastscan'] = node_lastscan
        else:
            d['lastscan'] = min(node_lastscan, disk_lastscan)

        return d

    def health(self, scan=False):
        """

        :param scan:
        :return: {}: 'cachestat', 'nodestat', 'diskstat', 'chunkstat'
        """
        stat = {}

        stat['cachestat'] = {}

        # nodestat
        stat['nodestat'] = self.instences[0].show()
        if not stat['nodestat']['running']:
            stat['diskstat'] = {}
            stat['chunkstat'] = {}
            return stat

        stat['diskstat'] = {}
        diskstat = os.path.join(self.config.shm, "nodectl/diskstat")
        if os.path.exists(diskstat):
            for pool in os.listdir(diskstat):
                stat_pool = os.path.join(diskstat, pool)
                if not os.path.isdir(stat_pool):
                    continue

                stat['diskstat'][pool] = {}
                for disk in os.listdir(stat_pool):
                    disk_num = disk.split('.')[0]
                    stat_file = os.path.join(stat_pool, disk)
                    if not os.path.exists(stat_file):
                        continue

                    if not stat_file.endswith('stat'):
                        continue

                    stat['diskstat'][pool][disk_num] = LichFile.read_dict(stat_file)
                    # print pool, disk_num, stat_file, stat['diskstat'][pool][disk_num]

        stat['chunkstat'] = {}
        chunkstat = os.path.join(self.config.shm, "nodectl/recovery")
        if os.path.exists(chunkstat):
            for pool in os.listdir(chunkstat):
                if not os.path.isdir(os.path.join(chunkstat, pool)):
                    continue

                stat['chunkstat'][pool] = {}

                recovery_pool = os.path.join(chunkstat, pool)
                immediately = os.path.join(recovery_pool, "immediately")
                if scan:
                    _exec_system("echo 1 > %s" % immediately, False)
                    #stat['cachestat'] = self.disk_manage.disk_check('cachestat')

                # interval = os.path.join(recovery_pool, "interval")

                stat['chunkstat'][pool] = {}
                stat['chunkstat'][pool] = self._merge_recovery_info(recovery_pool)
                stat['chunkstat'][pool]['scan_fail'] = 0

        return stat

    def addinstence(self, i):
        i = int(i)
        instence = None

        for x in self.instences:
            if x.service == i:
                instence = x
                break

        if instence is None:
            raise Exp(errno.ENOENT, 'not found instence %d' % (i))

        instence.init(self.config.clustername, instence.service, ttyonly=False)
        instence.start()

        instence_name = self.config.hostname + "/%s"%(str(instence.service))

        #todo, here need retry , just like cluster_meta.__addnode1()
        try:
            _exec_pipe([self.config.lich + "/libexec/lich.admin", "--addnode", instence_name])
        except Exp, e:
            instence.stop()
            raise Exp(e.errno, e.err)

    def cache_attach(self, args, verbose=False):
        self.disk_manage.cache_attach(args, verbose)

    def disk_meta(self, args, verbose=False):
        self.disk_manage.disk_meta(args, verbose)

    def disk_show(self, args, verbose=False):
        self.disk_manage.disk_show(args, verbose)

    def disk_load(self, args, verbose=False):
        self.disk_manage.disk_load(args, verbose=verbose, load_diff=True)

    def disk_set(self, arg):
        if arg not in ['clusteruuid', 'nodename', 'all']:
            raise Exp(errno.EINVAL, 'disk_set only support clusteruuid|nodename|all operate')
        self.disk_manage.disk_set(arg)

    def disk_check(self, arg):
        if arg not in ['cache', 'tier', 'writeback', 'speed', 'rotation', 'health']:
            raise Exp(errno.EINVAL, 'disk_check only support cache|tier|writeback|speed|rotation operate')
        self.disk_manage.disk_check(arg)

    def disk_list(self, is_all, cachedev, is_json, verbose):
        self.disk_manage.disk_list(is_all, cachedev, is_json, verbose)

    def disk_speed(self, devs):
        self.disk_manage.disk_speed(devs)

    def disk_add(self, devs, v, force, cache, pool, cachedev=None):
        rst_devs = []
        fail_list = []
        retry = 0
        core_devs = devs

        if cachedev is not None:
            while True:
                fail_list = self.bindcache(cachedev, core_devs, pool, force)
                if len(fail_list) and force and retry < 3:
                    retry = retry + 1
                    core_devs = fail_list
                    time.sleep(0.1)
                    continue
                else:
                    break

        if cachedev in devs:
            devs.remove(cachedev)

        for dev in devs:
            if dev in fail_list:
                continue

            mapping_dev = self.commoncache.get_mappingdev_by_coredev(dev)
            if mapping_dev is not None:
                if cachedev is not None:
                    rst_devs.append(mapping_dev)
                else:
                    if force:
                        self.commoncache.delcoredev_dangerously(dev)
                        rst_devs.append(dev)
                    else:
                        _derror("%s has cache, please remove it first or use --force !" % (dev))
                        fail_list.append(dev)
            else:
                #sdd是否可以加入集群,接口调用暂未考虑intelcas
                rst_devs.append(dev)

        if cachedev is not None:
            cset_uuid = self.commoncache.get_cset_uuid(cachedev)
        else:
            cset_uuid = None

        # TODO if reformat bcache cache/data disks, but stop here
        # next time, when we start lich, sometime it will add symlink
        # These reformatted disks will be viewed as OLD disks

        self.disk_manage.disk_add(rst_devs, v, force, cache, pool, cset_uuid=cset_uuid)
        if len(fail_list) > 0:
            exit(errno.EPERM)

    def disk_del(self, devs, v, force):
        dev_list = []
        for dev in devs:
            mapping_dev = self.commoncache.get_mappingdev_by_coredev(dev)
            if mapping_dev is not None:
                dev_list.append(mapping_dev)
            else:
                dev_list.append(dev)

        self.disk_manage.disk_del(dev_list, v)

    def raid_add(self, devs, force):
        self.disk_manage.raid_add(devs, force)

    def raid_del(self, devs, force):
        self.disk_manage.raid_del(devs, force)

    def raid_cache(self, switch, devs):
        if switch not in ['show', 'set']:
            raise Exp(errno.EINVAL, 'raid_cache only support show|set operate')

        policy = None
        for dev in devs:
            if switch == 'set' and dev == devs[-1]:
                policy = dev
                devs.remove(dev)
            elif not dev.startswith('/dev/'):
                raise Exp(errno.EINVAL, 'unknow device %s, device must start with /dev/' % dev)

        self.disk_manage.raid_cache(switch, devs, policy)

    def raid_miss(self):
        self.disk_manage.raid_miss()

    def raid_load(self):
        self.disk_manage.raid_load()

    def raid_flush(self):
        self.disk_manage.raid_flush()

    def disk_light(self, switch, devs):
        if switch not in ['start', 'stop', 'stat', 'list']:
            raise Exp(errno.EINVAL, 'disk_light only support start|stop|stat|list operate')
        self.disk_manage.disk_light(switch, devs)

    def dump_io(self):
        now = int(time.time())
        iodir = os.path.join(self.config.shm, 'volume', 'io')
        pooldirs = os.listdir(iodir)
        for pd in pooldirs:
            pooldir = os.path.join(iodir, pd)
            if os.path.isdir(pooldir):
                volinfos = os.listdir(pooldir)
                for vf in volinfos:
                    volinfo = os.path.join(pooldir, vf)
                    if os.path.isfile(volinfo):
                        mtime = os.path.getmtime(volinfo)
                        if now - mtime >=5:
                            os.remove(volinfo)

    def dump_clean(self):
        parent = '%s/tmp/dump/io' % (self.config.home)
        if not os.path.exists(parent):
            return

        now = int(time.time())
        for path in os.listdir(parent):
            dec = now - int(path)
            hold_time = (1 * 60 * 60)
            if dec > hold_time:
                full_path = os.path.join(parent, path)
                shutil.rmtree(full_path)

        _dmsg("dump io files clean ok")

    def remove(self, purge = False):
        dirs = []
        files = []

        configs = ["cgroup.conf", "cluster.conf", "hosts.conf", "lich.admin.conf", "lich.conf"]
        configs_abs = [os.path.join(self.config.home, 'etc', x) for x in configs]
        for x in configs_abs:
            files.append(x)

        app = os.path.join(self.config.home, 'lich')
        dirs.append(app)

        if purge:
            datas = [x.home for x in self.instences]
            subs = ["chunk", "node", "status", "dirty", "deleted", "skip", "fake"]
            subs_abs = []
            for x in datas:
                for y in subs:
                    subs_abs.append(os.path.join(x, y))

            for x in subs_abs:
                if os.path.isdir(x):
                    dirs.append(x)

                if os.path.isfile(x):
                    files.append(x)

        for x in dirs:
            try:
                print 'rmdir:', x
                shutil.rmtree(x)
            except OSError, e:
                if e.errno != errno.ENOENT:
                    pass

        for x in files:
            try:
                print 'remove:', x
                os.remove(x)
                pass
            except OSError, e:
                if e.errno != errno.ENOENT:
                    pass

    def metanode_balance(self):
        lfile = "/var/run/fusionstack_metanode_balance.lock"
        lock = _lock_file(lfile)

        is_operation = 0
        for i in self.instences:
            if (i.isadmin()):
                is_operation = 1
                balance()
                break
        if (is_operation == 0):
            raise Exp(errno.EPERM, 'The operation needs to be performed in admin node')

    def _iterator_all_lun(self, path, func):
        lun = []

        cmd = os.path.join(self.config.lich, "libexec/lichfs")
        try:
            (out_msg, err_msg) = _exec_pipe1([cmd, '--list', path], 1, False)
            for line in out_msg.split('\n'):
                if line == "":
                    break

                filetype = line.split(' ')[0]
                filename = line.split(' ')[8]
                abs_path = os.path.join(path, filename)
                if filetype.startswith('d'):
                    self._iterator_all_lun(abs_path, func)
                elif filetype.startswith('-'):
                    func(self.config.lich, abs_path)
                else:
                    _derror("lun:%s, unkown type:%s\n" % (abs_path, filetype))
                    continue
        except Exp, e:
            if e.errno == errno.ENOENT:
                return
            raise Exp(e.errno, e.err)

    def snapshot_manage(self):
        print "****************%s****************" % (time.strftime('%Y-%m-%d %H:%M:%S'))
        self._iterator_all_lun("/iscsi", _snapshot_manage)
        self._iterator_all_lun("/lichbd", _snapshot_manage)
        self._iterator_all_lun("/default", _snapshot_manage)
        print "****************snapshot manage ok !****************"

    def spare_check(self, v=False):
        bsize = 0
        blocks = 0
        bfree = 0
        w_size = 0
        e_size = 0
        msg = ''

        ins = self.instences[0]
        res = _exec_pipe([ins.lich_admin, '--stat', ins.name], p=False)
        res = _str2dict(res)
        admin = res['admin']
        if (admin != ins.name):
            return

        try:
            out, err = _exec_pipe1(['lichfs', '--fsstat'], 0, False)
        except Exp, e:
            raise Exp(e.errno, e.err)
        lines = out.strip().split('\n')
        for line in lines:
            if 'bsize' in line:
                bsize = int(line.split(':')[-1])
            if 'blocks' in line:
                blocks = int(line.split(':')[-1])
            if 'bfree' in line:
                bfree = int(line.split(':')[-1])
        fssize = bsize * blocks
        fsfree = bsize * bfree

        def __get_size_by_flag(flag, fssize):
            cmd = ['lichfs', '--attrget', '/']
            cmd.append(flag)
            try:
                out, err = _exec_pipe1(cmd, 0, False)
            except Exp, e:
                if 126 == e.errno:
                    return '', 0
                raise Exp(e.errno, e.err)

            str = out.strip()
            if str[-1] == '%':
                size = int(fssize * float(str[:-1]) * 0.01)
            else:
                size = _human_unreadable(str, True)
            return str, size

        w_str, w_size = __get_size_by_flag('spare_warn', fssize)
        e_str, e_size = __get_size_by_flag('spare_error', fssize)
        if fsfree < e_size:
            msg = "Spare Policy Error, error(%s), fs free(%s), fs(%s)" % (e_str, _human_readable(fsfree), _human_readable(fssize))
            _syserror(msg)
        elif fsfree < w_size:
            msg = "Spare Policy Warning, warning(%s), fs free(%s), fs(%s)" % (w_str, _human_readable(fsfree), _human_readable(fssize))
            _syswarn(msg)

        if v and len(msg):
            print msg

    def etcd(self, state, hosts):
        proxy = False

        array = hosts.split(',')
        if self.config.hostname not in array:
            proxy = True

        Etcd_manage.etcd_set(array, state, proxy)

    def clean_node_dangerously(self):
        _dwarn("clean node dangerously")

        assert '/opt/fusionstack' in self.config.home

        # TODO bcache hangup
        used_disk = self.disk_manage.list_all_used_disk()
        for disk in used_disk:
            try:
                if self.commoncache.is_running_coredev(disk):
                    self.commoncache.delcoredev_dangerously(disk)
            except Exp as e:
                traceback.print_exc()

        for disk in used_disk:
            try:
                if self.commoncache.is_running_cachedev(disk):
                    self.commoncache.delcachedev(disk, 0)
            except Exp as e:
                traceback.print_exc()

        # process
        os.system('pkill -9 lichd')
        os.system('pkill -9 redis-server')
        os.system('systemctl stop etcd')

        # file
        os.system('rm -rf %s/data/*' % self.config.home)
        os.system('rm -rf %s/core/*' % self.config.home)
        os.system('rm -rf %s/log/lich.log*' % self.config.home)
        os.system('rm -rf %s/etc/cluster.conf' % self.config.home)
        os.system('rm -rf /dev/shm/lich4')

    def bindcache(self, cachedev, coredevs, pool, force):
        fail_list = []

        for dev in coredevs:
            try:
                if self.config.cache_enable:
                    self.commoncache.bindcache(cachedev, dev, pool, force)
            except Exp, e:
                # traceback.print_exc()
                _derror("%s bind to cache %s fail, errmsg: %s" % (dev, cachedev, e.err))
                fail_list.append(dev)
                continue

        if len(fail_list) > 0:
            _derror("bind fail list:%s\n" % (fail_list))

        return fail_list

    def _check_disk_is_old(self, dev):
        all_used_disk = self.disk_manage.list_all_used_disk()
        if dev in all_used_disk:
            meta, _ = self.disk_manage.bdev.read_meta(dev)
            if meta['cset'] == "" or meta['cset'] == "None" :
                return True

        return False

    def cache_add(self, coredevs, cachedev, pool, force):
        cset_uuid = None
        dev_list = []
        ret = 0

        node_stat = self.instences[0].running()
        if node_stat is True:
            raise Exp(errno.EPERM, "Please stop the node service first if add cache !")

        if not self.config.cache_enable:
            _dwarn("cache disabled.\n")
            exit(errno.EPERM)

        for coredev in coredevs:
            if self._check_disk_is_old(coredev):
                _dwarn("dev:%s is old, not permit add cache, please check it!" % coredev)
                ret = errno.EPERM
                continue
            else:
                 dev_list.append(coredev)

        self.bindcache(cachedev, dev_list, pool, force)
        cset_uuid = self.commoncache.get_cset_uuid(cachedev)
        for coredev in dev_list:
            self.disk_manage.disk_setmeta_cset(coredev, cset_uuid, bcache_header=True)

        exit(ret)

    def cache_del(self, devs, flush=1):
        if flush:
            node_stat = self.instences[0].running()
            if node_stat is True:
                raise Exp(errno.EPERM, "Please stop the node service first if delete cache !")

        for dev in devs:
            if self.commoncache.is_running_coredev(dev):
                self.commoncache.delcoredev(dev, flush)
            elif self.commoncache.is_running_cachedev(dev):
                self.commoncache.delcachedev(dev, flush)
            else:
                _dwarn("%s no cache found, please check it!" % dev)

            print "dev %s cache delete ok!" % (dev)

    def setcachemode(self, dev, cache_mode, is_flush):
        self.commoncache.set_cache_mode(dev, cache_mode, is_flush)

    def setcachecutoff(self, dev, seq_cutoff):
        self.commoncache.set_cache_seq_cutoff(dev, seq_cutoff)

    def flushcachedev(self, cachedev, pool):
        self.commoncache.flush_cachedev(cachedev)

    def flushcoredev(self, coredevs, pool):
        for coredev in coredevs.split(','):
            self.commoncache.flush_coredev(coredev)

    def cacheset(self, key, value, cachedev=None):
        self.commoncache.cacheset(key, value, cachedev)

    def cacheget(self, key, cachedev=None):
        self.commoncache.cacheget(key, cachedev)

    def pool_gc(self, pools=[]):
        assert isinstance(pools, list)

        try:
            etcd_man = Etcd_manage()
            pools = etcd_man.pool_list()

            disk_man = DiskManage(self)
            return disk_man.pool_gc(pools)
        except Exp, e:
            _derror(e.err)

    def balance_chunk(self, runlevel=0):
        """
         {'volume-705b762c-d054-11e8-99c6-000c291d2532': {'fill_rate': '100',
                                                 'interval': '31536000',
                                                 'node': '8',
                                                 'speed': '0',
                                                 'status': 'done',
                                                 'step': '100',
                                                 'success': '10',
                                                 'thread': '10',
                                                 'total': '10',
                                                 'volume': '0'},
          'volume-a6d40e8a-d054-11e8-99c6-000c291d2532': {'fill_rate': '100',
                                                 'interval': '31536000',
                                                 'node': '8',
                                                 'speed': '0',
                                                 'status': 'done',
                                                 'step': '100',
                                                 'success': '10',
                                                 'thread': '10',
                                                 'total': '10',
                                                 'volume': '0'}}
        """
        d = {}

        prefix = os.path.join(self.config.shm, "nodectl/balance")
        if os.path.exists(prefix):
            for pool in os.listdir(prefix):
                if not os.path.isdir(os.path.join(prefix, pool)):
                    continue

                info_f = os.path.join(prefix, pool, 'info')

                if os.path.exists(info_f):
                    with open(info_f) as f:
                        s = f.read()
                        tmp = _str2dict(s)
                        d[pool] = {}
                        for k, v in tmp.iteritems():
                            if k not in ['status']:
                                d[pool][k] = int(v)
                            else:
                                d[pool][k] = v

        runlevel_path = os.path.join("/dev/shm/lich4/nodectl", "balance/runlevel")
        if runlevel == 1:
            _exec_system("echo 1 > %s" % runlevel_path, False)
        elif runlevel == 2:
            _exec_system("echo 2 > %s" % runlevel_path, False)
        else:
            pass

        return d


def usage(unhide):
    print ("usage:")
    print (sys.argv[0] + " --etcd [state] [host1,host2,host3 ...] ")
    print
    print (sys.argv[0] + " --stat [-v,--verbose] [--json]")
    print (sys.argv[0] + " --init")
    print (sys.argv[0] + " --start")
    print (sys.argv[0] + " --stop")
    print (sys.argv[0] + " --restart")
    print (sys.argv[0] + " --update [dist] [src] [backup]")
    print
    print (sys.argv[0] + " --disk_check cache|tier|writeback|speed|rotation")
    print (sys.argv[0] + " --disk_list [--cachedev <dev>] [--json]")
    print (sys.argv[0] + " --disk_add {dev1 dev2 ... | all [--force] [--pool pool] [--cachedev cachedev]}")
    print (sys.argv[0] + " --disk_del dev1 dev2 ...[--force]")
    print (sys.argv[0] + " --disk_light start|stop|stat|list dev1 dev2 ...")
    print
    print (sys.argv[0] + " --raid_add {inq1 inq2 ...[--force] | all}")
    print (sys.argv[0] + " --raid_del dev1 dev2 ...[--force]")
    print (sys.argv[0] + " --raid_cache show|set dev1 dev2 ...")
    print (" \t'[WT,WB,NORA,RA,ADRA,Cached,Direct,CachedBadBBU,NoCachedBadBBU,EnDskCache,DisDskCache]'")
    print (" \t'[Cached,R/W,Direct,EnDskCache,DisDskCache,smartpath]'")
    print
    print (sys.argv[0] + " --log {backup|clean|tail} [stime]")
    print (sys.argv[0] + " --log collect <stime> <etime>")
    print
    print (sys.argv[0] + " --disk2shm")
    print (sys.argv[0] + " --shm2disk")
    print
    print (sys.argv[0] + " --cache_add <dev1 dev2 ...> --cachedev <cachedev> --pool <pool> [--force]")
    print (sys.argv[0] + " --cache_del <dev1 dev2 ...> [--flush 0|1]" )
    print
    print (sys.argv[0] + " --flushcachedev <dev> --pool <pool>")
    print (sys.argv[0] + " --flushcoredev <dev1,dev2,...> --pool <pool>")
    print
    print (sys.argv[0] + " --cacheget <key> [--cachedev <cachedev>]")
    print (sys.argv[0] + " --cacheset <key> <value> [--cachedev <cachedev>]")
    print
    print (sys.argv[0] + " --setcachemode <writethrough|writeback|writearound|none> --cachedev <cachedev> [--flush 0|1]")
    print (sys.argv[0] + " --setcachecutoff <value> --cachedev <cachedev>")
    print

    if unhide:
        print (sys.argv[0] + " --vip_check")
        print (sys.argv[0] + " --spare_check [-v, --verbose]")
        print (sys.argv[0] + " --check [sysctl,iscsid]")
        print
        print (sys.argv[0] + " --health [scan | clean] [--force]")
        print (sys.argv[0] + " --scan")
        print
        # print (sys.argv[0] + " --metanode_balance")
        # print (sys.argv[0] + " --metabalance")
        # print (sys.argv[0] + " --chunkbalance, [--rand]")
        # print (sys.argv[0] + " --recover")
        # print (sys.argv[0] + " --recover_replica")
        # print
        print (sys.argv[0] + " --sethosts <ip> <hostname>")
        print (sys.argv[0] + " --delhosts <ip> <hostname>")
        print
        print (sys.argv[0] + " --addinstence instenceid")
        print (sys.argv[0] + " --drop [instenceid]")
        print
        print (sys.argv[0] + " --snapshot_manage")
        print (sys.argv[0] + " --setcron <0,1>")
        print (sys.argv[0] + " --skip <0,1>")
        #print (sys.argv[0] + " --remove [--purge]")
        print
        print (sys.argv[0] + " --raid_miss")
        print (sys.argv[0] + " --raid_load")
        print (sys.argv[0] + " --raid_flush")
        print
        print (sys.argv[0] + " --cache_attach")
        print
        print (sys.argv[0] + " --disk_meta")
        print (sys.argv[0] + " --disk_show")
        print (sys.argv[0] + " --disk_load")
        print ("\t[conf=configpath | meta=/dev/metadisk wlog=/dev/wlogdisk disk0..255=/dev/disk0..255dev]")
        print (sys.argv[0] + " --disk_set {clusteruuid|nodename|all}")
        print
        print (sys.argv[0] + " --unhide")


#@timeit()
def main():
    verbose = 0
    is_json = 0
    force = False
    cache = 0
    unhide = False
    op = ""
    ext = None
    rand = 0
    service = None
    ttyonly = True
    purge = False
    is_all = False
    cachedev = None
    coredev = None
    pool = None
    flush = 1
    cache_mode = ""

    try:
        opts, args = getopt.getopt(
            sys.argv[1:],
            'hvat', ['start', 'stop', 'restart', 'help', 'host', 'stat', 'force',
                     'health', 'init', 'check', 'update',
                     'metabalance', 'chunkbalance', 'metanode_balance',
                     'balance_chunk',
                     'disk2shm', 'shm2disk', 'cachedev=', 'coredev=', 'pool=', 'flush=',
                     'scan', 'recover', 'recover_replica', 'verbose', 'drop',
                     'setcron=', 'skip=', 'vip_check', 'ucarp', 'service=', 'json', 'log=',
                     'ttyonly', 'rand', 'sethosts','delhosts', 'addinstence',
                     'disk_meta', 'disk_show', 'disk_load',
                     'raid_load', 'raid_flush',
                     'disk_set=', 'disk_check=', 'disk_list', 'disk_add', 'disk_del', 'disk_light=',
                     'raid_add', 'raid_del', 'raid_cache=', 'raid_miss', 'disk_speed',
                     'cache_attach',
                     'pool_gc',
                     'dump_io', 'dump_clean', 'unhide', 'remove', 'purge',
                     'snapshot_manage', 'spare_check', 'etcd',
                     'clean_node_dangerously',
                     'bindcache', 'cache_add', 'cache_del', 'setcachemode=', 'setcachecutoff=',
                     'flushcachedev=', 'flushcoredev=', 'cacheset', 'cacheget'
                     ]
        )
    except getopt.GetoptError, err:
        print str(err)
        usage(unhide)
        exit(errno.EINVAL)

    # print 'args:', args

    newopts = copy.copy(opts)
    for o, a in opts:
        if o in ('-v', '--verbose'):
            verbose = 1
            newopts.remove((o, a))
        elif o in ('-a'):
            is_all = True
            newopts.remove((o, a))
        elif o in ('--json'):
            is_json = 1
            newopts.remove((o, a))
        elif o in ('--cachedev'):
            cachedev = a
            newopts.remove((o, a))
        elif o in ('--coredev'):
            coredev = a
            newopts.remove((o, a))
        elif o in ('--pool'):
            pool = a
            newopts.remove((o, a))
        elif o in ('--flush'):
            flush = a
            newopts.remove((o, a))
        elif o in ('--force'):
            force = True
            newopts.remove((o, a))
        elif (o == '--ttyonly'):
            ttyonly = True
            newopts.remove((o, a))
        elif (o == '--rand'):
            rand = 1
            newopts.remove((o, a))
        elif (o == '--purge'):
            purge = True
            newopts.remove((o, a))
        elif (o == '--unhide'):
            unhide = True
            usage(unhide)
            exit(0)

    try:
        node = Node()
        node.ttyonly = ttyonly
    except Exp, e:
        _derror(e.err)
        exit(e.errno)

    for o, a in newopts:
        if o in ('--help'):
            usage(unhide)
            exit(0)
        elif o == '--init':
            node.init()
        elif o == '--stat':
            op = "--stat"
            ext = args
        elif o == '--health':
            op = o
            if '--force' in args:
                force = True
                args.remove('--force')
        elif o == '--start':
            op = o
        elif o == '--stop':
            op = o
        elif o == '--restart':
            op = o
        elif o == '--disk2shm':
            op = o
        elif o == '--shm2disk':
            op = o
        elif o == '--check':
            if (len(sys.argv[2:])):
                node.check(op=sys.argv[2])
            else:
                node.check()
        elif o == '--update':
            node.update(sys.argv[2], sys.argv[3], sys.argv[4])
        elif o == '--metabalance':
            try:
                node.metabalance()
            except Exp, e:
                _derror(e)
                exit(e.errno)
        elif o == '--chunkbalance':
            op = o
            ext = "all"
        elif o == '--scan':
            op = o
        elif o == '--recover':
            op = o
        elif o == '--recover_replica':
            if (len(sys.argv[2:])):
                node.recover_replica(sys.argv[2])
            else:
                node.recover_replica()
        elif (o == '--drop'):
            if '--force' in args:
                force = True
                args.remove('--force')
            op = "--drop"
            if args:
                try:
                    assert(len(args) == 1)
                    ext = int(args[0])
                except Exception, e:
                    _derror('diskid was error, %s' % args)
                    exit(errno.EINVAL)
            else:
                ext = None
        elif (o == '--setcron'):
            op = o
            ext = a
        elif (o == '--skip'):
            op = o
            ext = a
        elif o == '--vip_check':
            op = o
        elif o == '--ucarp':
            op = o
        elif (o == '--service'):
            service = a
        elif (o == '--log'):
            op = o
            ext = a
            if (ext == 'backup'):
                try:
                    stime = sys.argv[3]
                except Exception, e:
                    stime = time.strftime('%Y%m%d-%H%M%S')
            elif (ext == 'collect'):
                if len(sys.argv) != 5:
                    _derror('Invalid argument')
                    print 'For example: '
                    print '     lich.node --log collect "2016-06-07 11:17:56" "2016-06-07 11:17:59"'
                    exit(errno.EINVAL)
            else:
                if len(args) != 0:
                    _derror('stime %s only used for backup' %args)
                    exit(errno.EINVAL)

        elif (o == '--ttyonly'):
            node.ttyonly = True
        elif (o == '--sethosts'):
            ip = sys.argv[2].strip()
            hostname = sys.argv[3].strip()
            try:
                etchosts_update(node.config.hosts_conf, ip, hostname)
            except Exp, e:
                _derror(e.err)
                exit(e.errno)
        elif (o == '--delhosts'):
            ip = sys.argv[2].strip()
            hostname = sys.argv[3].strip()
            try:
                etchosts_delete(node.config.hosts_conf, ip, hostname)
            except Exp, e:
                _derror(e.err)
                exit(e.errno)
        elif (o == '--addinstence'):
            op = o
            service = sys.argv[2].strip()
        elif (o == '--cache_attach'):
            op = o
            ext = args
        elif (o == '--disk_meta'):
            op = o
            ext = args
        elif (o == '--disk_show'):
            op = o
            ext = args
        elif (o == '--disk_load'):
            op = o
            ext = args
            # if len(ext) == 0:
            #     usage(unhide)
            #     exit(errno.EINVAL)
        elif (o == '--raid_load'):
            op = o
        elif (o == '--raid_flush'):
            op = o
        elif (o == '--disk_set'):
            op = o
            ext = a
        elif (o == '--disk_check'):
            op = o
            ext = a
        elif (o == '--disk_list'):
            op = o
        elif (o == '--disk_speed'):
            op = o
            ext = args
        elif (o == '--disk_add'):
            if '--force' in args:
                force = True
                args.remove('--force')
            if '--pool' in args:
                idx = args.index('--pool')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                service = args[idx + 1]
                args.remove(service)
                args.remove('--pool')

            if '--cache' in args:
                idx = args.index('--cache')
                if len(args) > idx + 1:
                    cache = int(args.pop(idx + 1))
                else:
                    cache = 100
                if cache > 100 or cache < 0:
                    _derror("cache range 0~100")
                    exit(errno.EINVAL)
                args.remove('--cache')

            if '--cachedev' in args:
                idx = args.index('--cachedev')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                cachedev = args[idx+1]
                args.remove(cachedev)
                args.remove('--cachedev')
            op = o
            ext = args
        elif (o == '--disk_del'):
            if '--force' in args:
                force = True
                args.remove('--force')
            op = o
            ext = args
        elif (o == '--raid_add'):
            if '--force' in args:
                force = True
                args.remove('--force')
            op = o
            ext = args
        elif (o == '--raid_del'):
            if '--force' in args:
                force = True
                args.remove('--force')
            op = o
            ext = args
        elif (o == '--raid_cache'):
            op = o
            service = a
            ext = args
        elif (o == '--raid_miss'):
            op = o
        elif (o == '--disk_light'):
            op = o
            service = a
            ext = args
        elif (o == '--dump_io'):
            node.dump_io()
        elif (o == '--dump_clean'):
            node.dump_clean()
        elif (o == '--remove'):
            op = o
        elif (o == '--metanode_balance'):
            op = o
        elif (o == '--snapshot_manage'):
            op = o
        elif (o == '--spare_check'):
            op = o
        elif (o == '--etcd'):
            op = o
        elif (o == '--clean_node_dangerously'):
            op = o
        elif o == '--balance_chunk':
            op = o
        elif (o == '--bindcache'):
            op = o
            ext = args
        elif (o == '--cache_add'):
            if '--force' in args:
                force = True
                args.remove('--force')

            if '--pool' in args:
                idx = args.index('--pool')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                pool = args[idx + 1]
                args.remove(pool)
                args.remove('--pool')

            if '--cachedev' in args:
                idx = args.index('--cachedev')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                cachedev = args[idx+1]
                args.remove(cachedev)
                args.remove('--cachedev')
            op = o
            ext = args
        elif (o == '--cache_del'):
            if '--flush' in args:
                idx = args.index('--flush')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                flush = args[idx+1]
                args.remove(flush)
                args.remove('--flush')
            op = o
            ext = args
        elif (o == '--setcachemode'):
            op = o
            ext = args
            cache_mode = a
        elif (o == '--setcachecutoff'):
            op = o
            ext = args
            seq_cutoff = a
        elif (o == '--flushcachedev'):
            op = o
            ext = args
            cachedev = a
            if cachedev is None or pool is None:
                usage(unhide)
                _derror('need enough param.')
                exit(errno.EINVAL)
        elif (o == '--flushcoredev'):
            op = o
            ext = args
            coredev = a
            if coredev is None or pool is None:
                usage(unhide)
                _derror('need enough param.')
                exit(errno.EINVAL)
        elif (o == '--cacheset'):
            op = o
            if '--cachedev' in args:
                idx = args.index('--cachedev')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                cachedev = args[idx+1]
                args.remove(cachedev)
                args.remove('--cachedev')
            ext = args
            if len(ext) != 2:
                _derror('need enough param.')
                exit(errno.EINVAL)
        elif (o == '--cacheget'):
            op = o
            if '--cachedev' in args:
                idx = args.index('--cachedev')
                if len(args) <= idx:
                    usage(unhide)
                    exit(errno.EINVAL)
                cachedev = args[idx+1]
                args.remove(cachedev)
                args.remove('--cachedev')
            ext = args
        elif o == '--pool_gc':
            op = o
            ext = args
        else:
            assert False, 'oops, unhandled option: %s, -h for help' % o
            exit(1)

    # RUN
    if op == "--stat":
        if 'check' in args:
            node.show_offline(False)
        elif 'scan' in args:
            node.show_offline(True)
        elif 'fullscan' in args:
            pass
        elif 'disk' in args:
            node.disk_show(args, verbose=verbose)
        elif is_json:
            node.show_json()
        elif len(args) > 0:
            try:
                # x: 'recovery' | 'msgctl', 'release' etc
                x = args[0]
                nstat = NodeStat()
                f = getattr(nstat, 'stat_%s' % x)
                f()
            except AttributeError, e:
                pass
        else:
            node.show(verbose)
    elif op == '--health':
        if 'clean' in args:
            try:
                node.disk_clean(force, False)
                _dmsg("clean disk ok")
            except Exp, e:
                _derror(e)
                _derror("clean disk fail")
                exit(e.errno)
        else:
            stat = node.health('scan' in args)
            if is_json:
                print json.dumps(stat)
            else:
                print json.dumps(stat, sort_keys=False, indent=4)
    elif (op == '--drop'):
        try:
            node.node_drop(ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif (op == '--setcron'):
        node.setcron(ext)
    elif (op == '--skip'):
        try:
            node.skip(ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif (op == '--vip_check'):
        node.vip_check()
    elif (op == '--ucarp'):
        node.ucarp_show()
    elif (op == '--log'):
        if (ext == 'backup'):
            node.log_backup(stime)
        elif (ext == 'clean'):
            node.log_clean()
        elif (ext == 'collect'):
            node.log_collect(sys.argv[3], sys.argv[4])
        elif (ext == 'tail'):
            res = node.log_tail()
            print (res)
        elif (ext == 'info'):
            log_info = node.log_info()
            print json.dumps(log_info, ensure_ascii=False)
        else:
            _derror(os.strerror(errno.EINVAL))
            usage(unhide)
            exit(errno.EINVAL)
    elif op == '--start':
        try:
            node.start(service, op)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--stop':
        node.stop(service, op)
    elif op == '--restart':
        node.stop(service, op)
        node.start(service, op)
    elif op == '--disk2shm':
        node.disk2shm(service, op)
    elif op == '--shm2disk':
        node.shm2disk(service, op)
    elif op == '--chunkbalance':
        node.recover_replica()
        node.chunkbalance(ext, rand=rand)
    elif op == '--addinstence':
        try:
            node.addinstence(service)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--scan':
        try:
            node.scan()
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--recover':
        node.recover(verbose=verbose)
    elif op == '--cache_attach':
        try:
            node.cache_attach(ext, verbose=verbose)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_meta':
        try:
            node.disk_meta(ext, verbose=verbose)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_show':
        try:
            node.disk_show(ext, verbose=verbose)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_load':
        try:
            node.disk_load(ext, verbose=verbose)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--raid_load':
        try:
            node.raid_load()
        except Exp, e:
            _derror(e.err)
            exit(e.errno)

        node.disk_manage.raid.raid_tool.raid_refresh()

        # node.stop(service, '--restart')
        # node.start(service, '--restart')
    elif op == '--raid_flush':
        try:
            node.raid_flush()
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_set':
        try:
            node.disk_set(ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_check':
        try:
            node.disk_check(ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_list':
        try:
            node.disk_list(is_all, cachedev, is_json, verbose)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_speed':
        try:
            node.disk_speed(ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--disk_add':
        if len(ext) == 0:
            usage(unhide)
            exit(errno.EINVAL)
        try:
            node.disk_add(ext, verbose, force, cache, service, cachedev)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--disk_del':
        if len(ext) == 0:
            usage(unhide)
            exit(errno.EINVAL)
        try:
            node.disk_del(ext, verbose, force)
            node.disk_clean(force, False)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--raid_add':
        if len(ext) == 0:
            usage(unhide)
            exit(errno.EINVAL)
        try:
            node.raid_add(ext, force)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--raid_del':
        try:
            node.raid_del(ext, force)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            _derror("raid delete fail. %s" % str(e.args))
            exit(e.args[0])
    elif op == '--raid_cache':
        try:
            node.raid_cache(service, ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--raid_miss':
        try:
            node.raid_miss()
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--disk_light':
        if service != 'list' and len(ext) == 0:
            usage(unhide)
            exit(errno.EINVAL)
        try:
            node.disk_light(service, ext)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
    elif op == '--bindcache':
        if cachedev is None or coredev is None or pool is None:
            _derror('need enough param.')
            exit(errno.EINVAL)
        try:
            node.bindcache(cachedev, coredev, pool, force)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--cache_add':
        if cachedev is None or len(ext) == 0:
            _derror('need enough param.')
            exit(errno.EINVAL)
        try:
            node.cache_add(ext, cachedev, pool, force)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--cache_del':
        if len(ext) == 0:
            _derror('need enough param.')
            exit(errno.EINVAL)

        try:
            node.cache_del(ext, flush)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == "--setcachemode":
        try:
            if cachedev is None:
                _derror("need input --cachedev <cachedev>")
                exit(errno.EINVAL)

            node.setcachemode(cachedev, cache_mode, flush)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == "--setcachecutoff":
        try:
            if cachedev is None:
                _derror("need input --cachedev <cachedev>")
                exit(errno.EINVAL)

            node.setcachecutoff(cachedev, seq_cutoff)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--flushcachedev':
        try:
            node.flushcachedev(cachedev, pool)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--flushcoredev':
        try:
            node.flushcoredev(coredev, pool)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--cacheset':
        try:
            node.cacheset(ext[0], ext[1], cachedev)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])
    elif op == '--cacheget':
        try:
            if ext:
                node.cacheget(ext[0], cachedev)
            else:
                node.cacheget('all', cachedev)
        except Exp, e:
            _derror(e.err)
            exit(e.errno)
        except Exception, e:
            raise
            _derror(e.args[1])
            exit(e.args[0])

    try:
        if op == '--etcd':
            node.etcd(sys.argv[2], sys.argv[3])
        elif op == '--pool_list':
            node.pool_list()
        elif op == '--pool_create':
            node.pool_create(service)
        elif op == '--pool_gc':
            node.pool_gc(ext)
        elif op == '--metanode_balance':
            node.metanode_balance()
        elif op == '--snapshot_manage':
            node.snapshot_manage()
        elif op == '--spare_check':
            node.spare_check(verbose)
        elif op == '--clean_node_dangerously':
            node.clean_node_dangerously()
        elif op == '--balance_chunk':
            runlevel = 0
            if len(sys.argv) > 2:
                if sys.argv[2] == 'start':
                    runlevel = 1
                elif sys.argv[2] == 'stop':
                    runlevel = 2
            d = node.balance_chunk(runlevel=runlevel)
            print json.dumps(d)
    except Exp, e:
        _derror(e.err)
        exit(e.errno)


if __name__ == '__main__':
    tlog.write(msg='%s' % sys.argv)

    #config = Config()
    if len(sys.argv) == 1:
        usage(False)
    else:
        main()
